df_train = readRDS("train.rds")	
#df_test = readRDS("test.rds")				#Read the train data


ord = order(df_train$f275,df_train$f276,df_train$f277, df_train$f521)	#Order by "loan id" and payment sequence
id = df_train$f277[ord]
id[is.na(id)]=0
loss_ind=as.integer(df_train$loss>0)[ord]
next_id = rep(NA, length(id))
next_id[length(id)] = 0
for(i in 1:length(id)-1) next_id[i] = id[i+1]
u=data.frame(id, next_id,seq_num=df_train$f521[ord], flag=(id!=next_id),loss_ind, df_train$loss[ord])
tapply(u$loss_ind, u$flag, mean)

#check
ord = order(df_train$f276,df_train$f521)
df_train = df_train[ord,]
df_train$f276[is.na(df_train$f276)]=0
next_f276 = rep(NA, nrow(df_train))
for(i in 1:nrow(df_train)-1) next_f276[i] = df_train$f276[i+1]
next_f276[nrow(df_train)] = 0
flag = (df_train$f276 != next_f276) 
tapply(df_train$loss>0, flag, mean)

df_subset = df_train[flag,]

sort(sapply(df_subset[,2:780],function(x) mean(df_subset$loss[x >= quantile(x, .999, na.rm=T)]>0)), decr=T)[2:15]

sort(sapply(df3[,2:780],function(x) mean(df3$loss[x <= quantile(x, .001)]>0)), decr=T)[1:10]


plot(tapply(df_subset$loss>0, df_subset$f674, mean))
mean((df_subset$loss>0)[df_subset$f674>60])

transaction_cnt=tapply(rep(1, nrow(df_train)), df_train$f276, sum)
transaction_id = tapply(df_train$f276, df_train$f276, max)
df_trans = data.frame(f276=transaction_id, transaction_cnt)

df_subset2 = merge(df_subset, df_trans, by="f276")
write.csv(df_subset2, "df_subset2.csv")

mean(df_subset2$loss>0)




ord = order(df_train$f274,df_train$f275,df_train$f276, df_train$f277, df_train$f521)	#Order by "loan id" and payment sequence
id = df_train$f274[ord] 
#f276 gives 0.046, 0.33.
#f275 gives 0.041 0.20
#f277 gives 0.0069 0.241
#what about pasting them together?
#f277 is cut off due to length - need to pull it in as a string
id[is.na(id)]=0
loss_ind=as.integer(df_train$loss>0)[ord]
next_id = rep(NA, length(id))
next_id[length(id)] = 0
for(i in 1:length(id)-1) next_id[i] = id[i+1]
u=data.frame(id, next_id,seq_num=df_train$f521[ord], flag=(id!=next_id),loss_ind)
tapply(u$loss_ind, u$flag, mean)


df_train=df_train[ord,]


#interesting:
#f521 f532 f274 f275 f276 f277 f521 f532 538 539 540 541 542 f543 544 545 551 553 554 557 471 loss

df2 = df_train[,c("f521", "f532", "f274", "f275", "f276", "f277", "f521", "f532", "f538", 
"f539", "f540", "f541", "f542", "f543", "f544", "f545", "f551", "f553", "f554", "f557", "f471", "loss"),]

edit(df2)

u=tapply(df2$loss>0, df2$f276, max)
v=tapply(df2$loss>0, df2$f276, length)
tmp=data.frame(u,v)
a=tapply(tmp$u, tmp$v, mean)
b=tapply(tmp$v, tmp$v, length)
mod = data.frame(cnt=b, mn=a)






u=tapply(df2$loss>0, df2$f276, max)
v=tapply(df2$loss>0, df2$f276, length)

sum(na.omit(df_train$f274)==2980)

df_test = readRDS("test.rds")

sum(na.omit(df_test$f276)==36631535491)



tapply(u$loss_ind, u$flag, mean)
#	     FALSE       TRUE 
#		 0.04047972  0.20663450 

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df_train = data.frame(lapply(df_train, mean_impute))
df_train = df_train[ord,]
df_subset = df_train[u$flag,]
mean(df_subset$loss>0)
length(df_subset$loss>0)
dim(df_subset)

quantilize = function(x)
{
	qntls = unique(quantile(x, seq(0, 1, .01)))
	print(length(qntls))
	if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(x)))
}


u = lapply(df_subset[,2:ncol(df_subset)-1], quantilize)
df_quant=data.frame(u, loss_ind=as.integer(df_subset$loss>0))

summary(glm(loss_ind~f2+f3+f4+f5+f6+f7+f8+f9+f10, data=df_quant, family=binomial))

plot(tapply(df_quant$loss_ind,df_quant$f21, mean) )

#f2 f7 f14 f15 f17 f21

m = glm(loss_ind~f2+f4+f13+f25+f67+f84+f94+f201+f211+f238+f340+f355+f376+
			     f404+f468+f471+f514+f515+f527+f528+f543+f588+f597+
			     f611+f615+f629+f637+f666+f670+f674+f675+f681+f683+f684+f686+f687+
				 f707+f715+f717+f721+f726+f727+f755+f772,
	data=df_quant, family=quasibinomial)

m2= glm(loss_ind~f2*f67 +f67+ f238+ f404+ f514+ f527+ f528+ f670+ f674+ f681, data=df_quant, family=binomial)

pred = predict(m, type="response")
plot(quantile(pred, seq(.01, .99, .01)), type="l")
a=cut(pred,quantile(pred, seq(0, 1, .01), include.lowest=T))
points(tapply(df_quant$loss_ind, a, mean))

summary(df_subset$f521)

quantile(df_subset$f471, .99)
mean(df_subset$loss[df_subset$f471>3]>0)
mean(df_subset$loss[df_subset$f468>1.8]>0)

sum(df_subset$f471>3)

sum(df_subset$f468>1.8)

sort(sapply(df_subset[,2:780],function(x) mean(df_subset$loss[x >= quantile(x, .999)]>0)), decr=T)[2:15]

